{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.randn([10, 10])\n",
    "y = torch.ones(10, dtype=torch.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = torch.nn.Linear(10,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = torch.optim.SGD([{'params':l.parameters(), 'lr':0.1}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt2 = torch.optim.SGD([{'params':l.parameters(), 'lr':-0.1}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = l(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = loss_fn(p, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0456,  0.2212,  0.2164, -0.2441,  0.1293,  0.0740, -0.0157, -0.1563,\n",
      "         -0.0669, -0.2609],\n",
      "        [ 0.1210,  0.3087, -0.2139,  0.1922,  0.1144,  0.3024, -0.1652, -0.1251,\n",
      "          0.0655, -0.2509]], requires_grad=True)\n",
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "opt.zero_grad()\n",
    "for z1 in l.parameters():\n",
    "    break\n",
    "print(z1)\n",
    "print(z1.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0456,  0.2212,  0.2164, -0.2441,  0.1293,  0.0740, -0.0157, -0.1563,\n",
      "         -0.0669, -0.2609],\n",
      "        [ 0.1210,  0.3087, -0.2139,  0.1922,  0.1144,  0.3024, -0.1652, -0.1251,\n",
      "          0.0655, -0.2509]], requires_grad=True)\n",
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "l.zero_grad()\n",
    "for z1 in l.parameters():\n",
    "    break\n",
    "print(z1)\n",
    "print(z1.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0456,  0.2212,  0.2164, -0.2441,  0.1293,  0.0740, -0.0157, -0.1563,\n",
      "         -0.0669, -0.2609],\n",
      "        [ 0.1210,  0.3087, -0.2139,  0.1922,  0.1144,  0.3024, -0.1652, -0.1251,\n",
      "          0.0655, -0.2509]], requires_grad=True)\n",
      "tensor([[ 0.0504,  0.1193,  0.0894,  0.0482,  0.2751, -0.1124,  0.1553,  0.0237,\n",
      "         -0.0620, -0.0553],\n",
      "        [-0.0504, -0.1193, -0.0894, -0.0482, -0.2751,  0.1124, -0.1553, -0.0237,\n",
      "          0.0620,  0.0553]])\n"
     ]
    }
   ],
   "source": [
    "loss.backward()\n",
    "for z2 in l.parameters():\n",
    "    break\n",
    "print(z2)\n",
    "print(z2.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0506,  0.2093,  0.2074, -0.2490,  0.1017,  0.0852, -0.0312, -0.1587,\n",
      "         -0.0607, -0.2553],\n",
      "        [ 0.1260,  0.3207, -0.2050,  0.1970,  0.1419,  0.2911, -0.1496, -0.1227,\n",
      "          0.0593, -0.2564]], requires_grad=True)\n",
      "tensor([[ 0.0504,  0.1193,  0.0894,  0.0482,  0.2751, -0.1124,  0.1553,  0.0237,\n",
      "         -0.0620, -0.0553],\n",
      "        [-0.0504, -0.1193, -0.0894, -0.0482, -0.2751,  0.1124, -0.1553, -0.0237,\n",
      "          0.0620,  0.0553]])\n"
     ]
    }
   ],
   "source": [
    "opt.step()\n",
    "for z3 in l.parameters():\n",
    "    break\n",
    "print(z3)\n",
    "print(z3.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0456,  0.2212,  0.2164, -0.2441,  0.1293,  0.0740, -0.0157, -0.1563,\n",
      "         -0.0669, -0.2609],\n",
      "        [ 0.1210,  0.3087, -0.2139,  0.1922,  0.1144,  0.3024, -0.1652, -0.1251,\n",
      "          0.0655, -0.2509]], requires_grad=True)\n",
      "tensor([[ 0.2078,  0.2485,  0.1716,  0.0826,  0.4510, -0.0205,  0.2592,  0.0583,\n",
      "         -0.0249, -0.0730],\n",
      "        [-0.2078, -0.2485, -0.1716, -0.0826, -0.4510,  0.0205, -0.2592, -0.0583,\n",
      "          0.0249,  0.0730]])\n"
     ]
    }
   ],
   "source": [
    "opt.step()\n",
    "for z3 in l.parameters():\n",
    "    break\n",
    "print(z3)\n",
    "print(z3.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
